{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.85 s, sys: 2.61 s, total: 5.46 s\n",
      "Wall time: 2.41 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/dev/malaya/malaya/tokenizer.py:202: FutureWarning: Possible nested set at position 3361\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n",
      "/home/ubuntu/dev/malaya/malaya/tokenizer.py:202: FutureWarning: Possible nested set at position 3879\n",
      "  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<module 'malaya' from '/home/ubuntu/dev/malaya/malaya/__init__.py'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import malaya\n",
    "malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sastrawi = malaya.stem.sastrawi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/huseinzol05/malay-dataset/blob/master/normalization/stemmer/stem.zip?raw=true\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/wiki-stem.json\n",
    "# !unzip stem.zip?raw=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['spider\\tspider',\n",
       " 'transeksual\\ttranseksual',\n",
       " 'khusus\\tkhusus',\n",
       " 'kriminologi\\tkriminologi',\n",
       " 'negeripingat\\tnegeripingat']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/home/ubuntu/malaya/stemmer-data-v5.txt') as fopen:\n",
    "    stemming = fopen.read().split('\\n')\n",
    "stemming[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41528"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left, right = [], []\n",
    "\n",
    "for s in stemming:\n",
    "    l, r = s.split('\\t')\n",
    "    left.append(l)\n",
    "    right.append(r)\n",
    "    \n",
    "len(left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('/home/ubuntu/malaya/news-stem.json') as fopen:\n",
    "    news = json.load(fopen)\n",
    "    \n",
    "for s in news:\n",
    "    left.append(s[0])\n",
    "    right.append(s[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/ubuntu/malaya/wiki-stem.json') as fopen:\n",
    "    wiki = json.load(fopen)\n",
    "    \n",
    "for s in wiki:\n",
    "    left.append(s[0])\n",
    "    right.append(s[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1832338, 1832338)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('transeksual', 'spider')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left[1], right[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'transeksual'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sastrawi.stem(f'{left[1]}lah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sakit'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem = sastrawi.stem('sakitnya')\n",
    "stem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "hujung = {\n",
    "    'kan': ['kann', 'kkan', 'kaann', 'kn', 'kkn'],\n",
    "    'lah': ['lh', 'lhh', 'lhaa', 'lha', 'laah', 'laahh', 'ler', 'la'],\n",
    "    'nya': ['nye', 'ny', 'nyee', 'nyye', 'nyo', 'nyoo'],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import random\n",
    "\n",
    "vowels = 'aeiou'\n",
    "consonants = 'bcdfghjklmnpqrstvwxyz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_hubung(word, p_remove_vowels = 0.5, p_duplicate = 0.7, min_duplicate = 2, max_duplicate = 3):\n",
    "    word = ['' if c in vowels and random.random() > p_remove_vowels else c for c in word]\n",
    "    word = list(filter(None, word))\n",
    "    for i in range(len(word)):\n",
    "        w = word[i]\n",
    "        if random.random() > p_duplicate:\n",
    "            duplicate = random.randint(min_duplicate, max_duplicate)\n",
    "            word[i] = w * duplicate\n",
    "    return ''.join(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sakit'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "malaya.augmentation.vowel_alternate('sakit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'teerrr'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augment_hubung('ter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('ter', 'nya')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'tersakitnya'\n",
    "stemmed = sastrawi.stem(string)\n",
    "index = [(m.start(0), m.end(0)) for m in re.finditer(stemmed, string, flags=re.IGNORECASE)][0]\n",
    "permulaan = string[:index[0]]\n",
    "hujungan = string[index[1]:]\n",
    "permulaan, hujungan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "awalan = ['pe', 'pem', 'pen', 'peng', 'pel', 'ke', 'juru', 'pra', 'dwi',\n",
    "         'me', 'memper', 'ber', 'ter', 'di', 'diper', 'ter', 'te', 'se',\n",
    "         'mem', 'diper']\n",
    "\n",
    "akhiran = ['lah', 'an', 'kan', 'nya', 'wan', 'wati']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nyo'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.choice(hujung.get(hujungan, [hujungan]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hamba'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sastrawi.stem('memperhambakan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_left = set(left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 63838/63838 [00:03<00:00, 18127.31it/s]\n"
     ]
    }
   ],
   "source": [
    "new_left, new_right = [], []\n",
    "for w in tqdm(malaya.dictionary.bahasa.dbp.words | malaya.dictionary.bahasa.cambridge_words.words | malaya.dictionary.bahasa.kamus_dewan.words | malaya.dictionary.bahasa.words.words):\n",
    "    if w in set_left:\n",
    "        continue\n",
    "    s = sastrawi.stem(w)\n",
    "    new_left.append(w)\n",
    "    new_right.append(s)\n",
    "    set_left.add(w)\n",
    "    \n",
    "left.extend(new_left)\n",
    "right.extend(new_right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1855109, 1855109)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(list('nn')).shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment(word, p_augment_word = 0.7):\n",
    "    awal = random.choice(awalan)\n",
    "    akhir = random.choice(akhiran)\n",
    "    joined = awal + word + akhir\n",
    "    stemmed = sastrawi.stem(joined)\n",
    "    if stemmed == joined:\n",
    "        return word, word\n",
    "    \n",
    "    index = [(m.start(0), m.end(0)) for m in re.finditer(stemmed, joined, flags=re.IGNORECASE)][0]\n",
    "    permulaan = joined[:index[0]]\n",
    "    hujungan = joined[index[1]:]\n",
    "    \n",
    "    if not len(permulaan) and not len(hujungan):\n",
    "        return False, left, right\n",
    "    \n",
    "    print(joined, stemmed, permulaan, hujungan)\n",
    "    \n",
    "    augmented_permulaan = augment_hubung(permulaan)\n",
    "    if len(augmented_permulaan) > 1:\n",
    "        permulaan = augmented_permulaan\n",
    "    \n",
    "    augmented_hujungan = augment_hubung(hujungan)\n",
    "    if len(augmented_hujungan) > 1:\n",
    "        hujungan = augmented_hujungan\n",
    "        \n",
    "    if random.random() > p_augment_word:\n",
    "        word = malaya.augmentation.vowel_alternate(word)\n",
    "    \n",
    "    return True, permulaan + word + hujungan, word\n",
    "\n",
    "def augment_hujung(word, p_augment_word = 0.7):\n",
    "    akhir = random.choice(akhiran)\n",
    "    joined = word + akhir\n",
    "    stemmed = sastrawi.stem(joined)\n",
    "    if stemmed == joined:\n",
    "        return False, word, word\n",
    "    \n",
    "    index = [(m.start(0), m.end(0)) for m in re.finditer(stemmed, joined, flags=re.IGNORECASE)][0]\n",
    "    hujungan = joined[index[1]:]\n",
    "    \n",
    "    if not len(hujungan):\n",
    "        return False, word, word\n",
    "    \n",
    "    if hujungan in hujung:\n",
    "        hujungan = random.choice(hujung.get(hujungan, [hujungan]))\n",
    "    else:\n",
    "        augmented_hujungan = augment_hubung(hujungan)\n",
    "        if len(augmented_hujungan) > 1:\n",
    "            hujungan = augmented_hujungan\n",
    "            \n",
    "    if np.unique(list(hujungan)).shape[0] == 1:\n",
    "        return False, word, word\n",
    "        \n",
    "    if random.random() > p_augment_word:\n",
    "        word = malaya.augmentation.vowel_alternate(word)\n",
    "    \n",
    "    return True, word + hujungan, word\n",
    "\n",
    "def augment_left_right(left, right, augment_hubungan = True, p_augment_word = 0.7):\n",
    "    index = [(m.start(0), m.end(0)) for m in re.finditer(right, left, flags=re.IGNORECASE)]\n",
    "    permulaan = left[:index[0][0]]\n",
    "    hujungan = left[index[-1][1]:]\n",
    "\n",
    "    not_right = left[index[0][0]:index[-1][1]]\n",
    "    # print(permulaan, hujungan, not_right)\n",
    "    \n",
    "    if not len(permulaan) and not len(hujungan):\n",
    "        return False, left, right\n",
    "    \n",
    "    if augment_hubungan:\n",
    "        augmented_permulaan = augment_hubung(permulaan)\n",
    "        if len(augmented_permulaan) > 1:\n",
    "            permulaan = augmented_permulaan\n",
    "            \n",
    "    if np.unique(list(permulaan)).shape[0] == 1:\n",
    "        return False, left, right\n",
    "    \n",
    "    if augment_hubungan:\n",
    "        augmented_hujungan = augment_hubung(hujungan)\n",
    "        if len(augmented_hujungan) > 1:\n",
    "            hujungan = augmented_hujungan\n",
    "            \n",
    "    if np.unique(list(hujungan)).shape[0] == 1:\n",
    "        return False, left, right\n",
    "    \n",
    "    if random.random() > p_augment_word and not_right == right:\n",
    "        right = malaya.augmentation.vowel_alternate(right)\n",
    "        selected = right\n",
    "    else:\n",
    "        selected = not_right\n",
    "    \n",
    "    joined = permulaan + selected + hujungan\n",
    "    joined = re.sub(r'^-+|-+$', '-', joined).strip()\n",
    "    \n",
    "    \n",
    "    if joined == left:\n",
    "        result = False\n",
    "    else:\n",
    "        result = True\n",
    "    \n",
    "    return result, joined, right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['spider',\n",
       " 'transeksual',\n",
       " 'khusus',\n",
       " 'kriminologi',\n",
       " 'negeripingat',\n",
       " 'melingkungi',\n",
       " 'terlajak',\n",
       " 'wooi',\n",
       " 'penguin',\n",
       " 'agung']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(True, 'agungnyo', 'agung')\n",
      "(True, 'agungan', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungaannn', 'agung')\n",
      "(True, 'agungnyoo', 'agung')\n",
      "(True, 'agngkkn', 'agng')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungny', 'agung')\n",
      "(True, 'agnglhh', 'agng')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungan', 'agung')\n",
      "(True, 'agungkkn', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agnglaah', 'agng')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungnyee', 'agung')\n",
      "(True, 'agungkaann', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungaann', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agunglaahh', 'agung')\n",
      "(True, 'agungkkn', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agunglhh', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungnye', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungkann', 'agung')\n",
      "(True, 'agungnyo', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungan', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungnyoo', 'agung')\n",
      "(True, 'agngla', 'agng')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungkn', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungkkn', 'agung')\n",
      "(True, 'agungkaann', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agungnyo', 'agung')\n",
      "(False, 'agung', 'agung')\n",
      "(True, 'agunglhaa', 'agung')\n",
      "(True, 'agungan', 'agung')\n"
     ]
    }
   ],
   "source": [
    "for i in range(50):\n",
    "    print(augment_hujung('agung'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(True, 'mnganggapnya', 'anggap')\n",
      "(True, 'mmeeenganggapnnnyaaa', 'anggap')\n",
      "(True, 'mnnngganggapnnyaaa', 'anggap')\n",
      "(True, 'mnganggapnyy', 'anggap')\n",
      "(True, 'mmmennnganggapny', 'anggap')\n",
      "(True, 'menganggapny', 'anggap')\n",
      "(True, 'mnnnganggapnya', 'anggap')\n",
      "(True, 'mmnganggpnny', 'anggp')\n",
      "(True, 'mmnnnganggapnyaa', 'anggap')\n",
      "(True, 'mmmnganggapnyy', 'anggap')\n",
      "(True, 'menganggapnnnyya', 'anggap')\n",
      "(True, 'menganggapnnnya', 'anggap')\n",
      "(True, 'mnnnganggapnyyyaaa', 'anggap')\n",
      "(True, 'mmenggganggapnya', 'anggap')\n",
      "(True, 'meenngganggapnnyyya', 'anggap')\n",
      "(True, 'mnganggapnya', 'anggap')\n",
      "(True, 'mnganggapnyaa', 'anggap')\n",
      "(True, 'mnnngganggapny', 'anggap')\n",
      "(True, 'mmenganggapnya', 'anggap')\n",
      "(True, 'mmmnggganggpnyyya', 'anggp')\n",
      "(True, 'mnggganggapnyyya', 'anggap')\n",
      "(True, 'mmmenganggapnya', 'anggap')\n",
      "(True, 'meeenganggapnyyya', 'anggap')\n",
      "(True, 'meeengganggapnyyy', 'anggap')\n",
      "(True, 'meennggganggapny', 'anggap')\n",
      "(True, 'mmnganggapny', 'anggap')\n",
      "(True, 'mmmnganggapnny', 'anggap')\n",
      "(True, 'menganggapnyyya', 'anggap')\n",
      "(True, 'mengganggapny', 'anggap')\n",
      "(True, 'mmmeeengganggpnnya', 'anggp')\n",
      "(True, 'mnnnganggapnyaaa', 'anggap')\n",
      "(True, 'menganggpny', 'anggp')\n",
      "(True, 'mnnganggapnyyyaa', 'anggap')\n",
      "(True, 'meeenggganggapny', 'anggap')\n",
      "(True, 'mmnnnganggapnnya', 'anggap')\n",
      "(True, 'mmenganggpnnyy', 'anggp')\n",
      "(True, 'mnnganggapnyyy', 'anggap')\n",
      "(True, 'mmnganggapnya', 'anggap')\n",
      "(False, 'menganggapnya', 'anggap')\n",
      "(True, 'menganggapnnnya', 'anggap')\n",
      "(True, 'mmmenganggpnnyy', 'anggp')\n",
      "(True, 'menggganggapny', 'anggap')\n",
      "(True, 'mnganggapnya', 'anggap')\n",
      "(True, 'mngganggapnnny', 'anggap')\n",
      "(True, 'menggganggpnnny', 'anggp')\n",
      "(True, 'menganggapnyaaa', 'anggap')\n",
      "(True, 'mmenganggapny', 'anggap')\n",
      "(True, 'mnggganggapnnny', 'anggap')\n",
      "(True, 'mennnganggpnnyaaa', 'anggp')\n",
      "(True, 'mmnnnganggapnnnyyaa', 'anggap')\n"
     ]
    }
   ],
   "source": [
    "for i in range(50):\n",
    "    print(augment_left_right('menganggapnya', 'anggap'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def cleaning(word):\n",
    "#     c = re.sub(r'^-+|-+$', '-', word.lower()).strip().replace('\"', ' ').replace('\\'', ' ')\n",
    "#     c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "#     if len(c) and c[-1] in '-':\n",
    "#         c = c[:-1]\n",
    "#         c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "#     if len(c) and c[0] in '-':\n",
    "#         c = c[1:]\n",
    "#         c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "#     return c\n",
    "\n",
    "# left_, right_ = [], []\n",
    "# for i in range(len(left)):\n",
    "#     c_l = cleaning(left[i])\n",
    "#     if len(c_l) > 1 and c_l[-1] not in '-' and c_l[0] not in '-' and ' ' not in c_l and c_l.count('-') <= 1 and all([len(a) > 1 for a in c_l.split('-')]):\n",
    "#         left_.append(c_l)\n",
    "#     c_r = cleaning(right[i])\n",
    "#     if len(c_r) > 1 and c_r[-1] not in '-' and c_r[0] not in '-' and ' ' not in c_r and c_r.count('-') <= 1 and all([len(a) > 1 for a in c_l.split('-')]):\n",
    "#         right_.append(c_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorted_ = sorted(list(set(left_ + right_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('unique-words.json', 'w') as fopen:\n",
    "#     json.dump(sorted_, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaning(word):\n",
    "    c = re.sub(r'^-+|-+$', '-', word).strip().replace('\"', ' ').replace('\\'', ' ')\n",
    "    c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "    if len(c) and c[-1] in '-':\n",
    "        c = c[:-1]\n",
    "        c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "    if len(c) and c[0] in '-':\n",
    "        c = c[1:]\n",
    "        c = re.sub(r'[ ]+', ' ', c).strip()\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 1855109/1855109 [00:02<00:00, 658773.84it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(left))):\n",
    "    left[i] = cleaning(left[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 1855109/1855109 [00:01<00:00, 1700055.63it/s]\n"
     ]
    }
   ],
   "source": [
    "combined = []\n",
    "for i in tqdm(range(len(left))):\n",
    "    if ' ' in left[i]:\n",
    "        continue\n",
    "    if len(left[i]) < 2:\n",
    "        continue\n",
    "    if len(right[i]) < 2:\n",
    "        continue\n",
    "    if left[i].count('-') > 1:\n",
    "        continue\n",
    "    if right[i][0] in '-.':\n",
    "        continue\n",
    "    if right[i][-1] in '-.':\n",
    "        continue\n",
    "    combined.append(f'{left[i]}<<>>{right[i]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "for c in set(combined):\n",
    "    if '<<>>' in c:\n",
    "        l, r = c.split('<<>>')\n",
    "        filtered_left.append(l)\n",
    "        filtered_right.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1228060, 1796170, 1855109)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_left), len(combined), len(left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(filtered_left, filtered_right, test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train_stem.json', 'w') as fopen:\n",
    "    json.dump({'X': train_X, 'Y': train_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test_stem.json', 'w') as fopen:\n",
    "    json.dump({'X': test_X, 'Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 1855109/1855109 [00:03<00:00, 586507.14it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(198320, 0.10690476947715741)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_same = []\n",
    "for i in tqdm(range(len(left))):\n",
    "    left_ = left[i].lower()\n",
    "    left_ = re.sub(r'^-+|-+$', '-', left_).strip().replace('\"', ' ').replace('\\'', ' ')\n",
    "    left_ = re.sub(r'[ ]+', ' ', left_).strip()\n",
    "    right_ = right[i].lower()\n",
    "    \n",
    "    if left_ != right_ and left[i].lower() != right[i].lower():\n",
    "        left_ = cleaning(left[i])\n",
    "        not_same.append((left_, right[i]))\n",
    "        \n",
    "len(not_same), len(not_same) / len(left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('cacingan', 'cacing'),\n",
       " ('pertaruh', 'taruh'),\n",
       " ('bering-bering', 'ring'),\n",
       " ('mengukirnya', 'ukir'),\n",
       " ('menciut-ciut', 'ciut'),\n",
       " ('mentakwilkan', 'takwil'),\n",
       " ('dikepal-kepal', 'kepal'),\n",
       " ('menggaduk', 'gaduk'),\n",
       " ('berkeritik-keritik', 'keritik'),\n",
       " ('bersudah', 'sudah'),\n",
       " ('terpaling-paling', 'paling'),\n",
       " ('berbedan', 'bedan'),\n",
       " ('menengil', 'tengil'),\n",
       " ('didongengkan', 'dongeng'),\n",
       " ('guntang-guntang', 'guntang'),\n",
       " ('berpiak', 'piak'),\n",
       " ('menggeramkan', 'geram'),\n",
       " ('bercelur', 'celur'),\n",
       " ('penjajaan', 'jaja'),\n",
       " ('tersekang', 'sekang'),\n",
       " ('mendangkalkan', 'dangkal'),\n",
       " ('berperdua', 'dua'),\n",
       " ('meligih', 'ligih'),\n",
       " ('nipiskan', 'nipis'),\n",
       " ('bertongkang', 'tongkang'),\n",
       " ('kegelian', 'geli'),\n",
       " ('berhitung-hitung', 'hitung'),\n",
       " ('bercerai-ceraian', 'cerai'),\n",
       " ('bertetar', 'tetar'),\n",
       " ('sampai-menyampai', 'sampai'),\n",
       " ('menyendat', 'sendat'),\n",
       " ('menyenak', 'senak'),\n",
       " ('lingkah', 'ling'),\n",
       " ('melesang', 'lesang'),\n",
       " ('kehangitan', 'hangit'),\n",
       " ('membuayai', 'buaya'),\n",
       " ('setarafkan', 'taraf'),\n",
       " ('menggelambir', 'gelambir'),\n",
       " ('memfanakan', 'fana'),\n",
       " ('mendiani', 'ani'),\n",
       " ('mengesip', 'kesip'),\n",
       " ('bungkusnya', 'bungkus'),\n",
       " ('melembang', 'lembang'),\n",
       " ('topangan', 'topang'),\n",
       " ('berlindungkan', 'lindung'),\n",
       " ('terserelum', 'rum'),\n",
       " ('difanakan', 'fana'),\n",
       " ('pelekah', 'lekah'),\n",
       " ('mengalem', 'alem'),\n",
       " ('memantaskan', 'pantas'),\n",
       " ('terbeti', 'beti'),\n",
       " ('menghutangi', 'hutang'),\n",
       " ('penspekulasi', 'spekulasi'),\n",
       " ('menjantang', 'jantang'),\n",
       " ('bergaruk-garuk', 'garuk'),\n",
       " ('tetak-menetak', 'tetak'),\n",
       " ('memperkuda', 'kuda'),\n",
       " ('bertahuan', 'tahu'),\n",
       " ('tisikan', 'tisik'),\n",
       " ('sepak-menyepak', 'sepak'),\n",
       " ('berdakar-dakar', 'dakar'),\n",
       " ('terselusur', 'selusur'),\n",
       " ('lihat-melihat', 'lihat'),\n",
       " ('dan-dan', 'dan'),\n",
       " ('menyengapkan', 'sengap'),\n",
       " ('palas-palas', 'palas'),\n",
       " ('berendul', 'endul'),\n",
       " ('pengasimilasian', 'asimilasi'),\n",
       " ('mengepunyai', 'punya'),\n",
       " ('pedap', 'dap'),\n",
       " ('terjumlahkan', 'jumlah'),\n",
       " ('menghinap-hinap', 'hinap'),\n",
       " ('lungsuran', 'lungsur'),\n",
       " ('mendinamit', 'dinamit'),\n",
       " ('menujumkan', 'nujum'),\n",
       " ('mererot', 'rerot'),\n",
       " ('menyendal menyendalkan', 'sendal sendal'),\n",
       " ('mencecap', 'cecap'),\n",
       " ('kebukuan', 'buku'),\n",
       " ('tersengguk', 'sengguk'),\n",
       " ('ujar-ujaran', 'ujar'),\n",
       " ('singsingkan', 'singsing'),\n",
       " ('ceratan', 'cerat'),\n",
       " ('mendidis', 'didis'),\n",
       " ('bergulung-gulung', 'gulung'),\n",
       " ('bergaru', 'garu'),\n",
       " ('mengokok', 'okok'),\n",
       " ('kekacip', 'kacip'),\n",
       " ('mengarwahkan', 'arwah'),\n",
       " ('menyeraya', 'seraya'),\n",
       " ('pungkahan', 'pungkah'),\n",
       " ('menggayut', 'gayut'),\n",
       " ('berlapih', 'lapih'),\n",
       " ('mengayaikan', 'kayai'),\n",
       " ('meloya', 'loya'),\n",
       " ('sejenang', 'jenang'),\n",
       " ('menyeratakan', 'rata'),\n",
       " ('sepion', 'pion'),\n",
       " ('menyentung', 'sentung'),\n",
       " ('ditumangkan', 'tumang'),\n",
       " ('lantakan', 'lantak'),\n",
       " ('menderu-deru', 'deru'),\n",
       " ('mencepol', 'cepol'),\n",
       " ('terjerongkong', 'jerongkong'),\n",
       " ('berabarkan', 'abar'),\n",
       " ('membising', 'bising'),\n",
       " ('bertopang', 'topang'),\n",
       " ('busuk-busuk', 'busuk'),\n",
       " ('perbahasa', 'bahasa'),\n",
       " ('perundung', 'rundung'),\n",
       " ('selaputnya', 'selaput'),\n",
       " ('memeriksai', 'periksa'),\n",
       " ('memepas', 'pepas'),\n",
       " ('melabai', 'laba'),\n",
       " ('berdegup-degup', 'degup'),\n",
       " ('pepayang', 'payang'),\n",
       " ('penghausan', 'haus'),\n",
       " ('berderis', 'deris'),\n",
       " ('membagaikan', 'bagai'),\n",
       " ('berdetasan', 'detas'),\n",
       " ('mencamuk', 'camuk'),\n",
       " ('peniat', 'niat'),\n",
       " ('kesabahan', 'sabah'),\n",
       " ('tertindis', 'tindis'),\n",
       " ('menyelawatkan', 'selawat'),\n",
       " ('secerdik-cerdik', 'cerdik'),\n",
       " ('memperburukkan', 'buruk'),\n",
       " ('memperduakan', 'dua'),\n",
       " ('mencuil', 'cuil'),\n",
       " ('tikamkan', 'tikam'),\n",
       " ('penangkalan', 'tangkal'),\n",
       " ('kerap-kerap', 'kerap'),\n",
       " ('jijiknya', 'jijik'),\n",
       " ('jahit-menjahit', 'jahit'),\n",
       " ('dipale', 'pale'),\n",
       " ('mencabik-cabikkan', 'cabik'),\n",
       " ('ditempeli', 'tempel'),\n",
       " ('bertahana', 'tahana'),\n",
       " ('kekeruh', 'keruh'),\n",
       " ('gamkan', 'gam'),\n",
       " ('terpelosot', 'pelosot'),\n",
       " ('menginspirasikan', 'inspirasi'),\n",
       " ('peradang', 'adang'),\n",
       " ('mengabur', 'abur'),\n",
       " ('menjaraki', 'jarak'),\n",
       " ('berlonggokan', 'longgok'),\n",
       " ('berkelipat', 'kelipat'),\n",
       " ('menekang', 'tekang'),\n",
       " ('mengejap', 'kejap'),\n",
       " ('beretetan', 'retet'),\n",
       " ('dimasyarakatkan', 'masyarakat'),\n",
       " ('memenget-menget', 'menget'),\n",
       " ('mempersandang', 'sandang'),\n",
       " ('mempergantungi', 'gantung'),\n",
       " ('mendali', 'dal'),\n",
       " ('menggolong-golongkan', 'golong'),\n",
       " ('berapun', 'bera'),\n",
       " ('pesawangan', 'sawang'),\n",
       " ('bergeser-geser', 'geser'),\n",
       " ('pencemas', 'cemas'),\n",
       " ('mengibadatkan', 'ibadat'),\n",
       " ('mengoja', 'oja'),\n",
       " ('bertembak-tembakan', 'tembak'),\n",
       " ('tumbukkan', 'tumbuk'),\n",
       " ('menatangkan', 'natang'),\n",
       " ('kesangatan', 'sangat'),\n",
       " ('penimbun', 'timbun'),\n",
       " ('menghamburi', 'hambur'),\n",
       " ('menyabungkan', 'sabung'),\n",
       " ('memparadam', 'paradam'),\n",
       " ('geruai', 'gua'),\n",
       " ('memetiki', 'tik'),\n",
       " ('kedadah', 'dadah'),\n",
       " ('pentilan', 'pentil'),\n",
       " ('terlancang', 'lancang'),\n",
       " ('sepenggalah', 'pengga'),\n",
       " ('bergendakan', 'gendak'),\n",
       " ('membesit', 'besit'),\n",
       " ('mengobak', 'kobak'),\n",
       " ('persebatian', 'bati'),\n",
       " ('merekat', 'rekat'),\n",
       " ('ditangguk', 'tangguk'),\n",
       " ('setubuhi', 'tubuh'),\n",
       " ('rembukan', 'rembuk'),\n",
       " ('tinju-meninju', 'tinju'),\n",
       " ('mencekukkan', 'cekuk'),\n",
       " ('kelebak', 'lebak'),\n",
       " ('mengimplisitkan', 'implisit'),\n",
       " ('memasakkannya', 'masak'),\n",
       " ('umpankan', 'umpan'),\n",
       " ('senyamuk', 'nyamuk'),\n",
       " ('menjebolkan', 'jebol'),\n",
       " ('terbentur', 'bentur'),\n",
       " ('mengiraikan', 'kirai'),\n",
       " ('menyulami', 'sulam'),\n",
       " ('membabaskan', 'babas'),\n",
       " ('berpapah', 'papah'),\n",
       " ('mendeliki', 'delik'),\n",
       " ('kebarat-baratan', 'barat'),\n",
       " ('beralit', 'alit'),\n",
       " ('berpenaruhan', 'taruh'),\n",
       " ('menjolek', 'jolek'),\n",
       " ('menggayutkan', 'gayut'),\n",
       " ('persalin', 'salin'),\n",
       " ('bertekung', 'tekung'),\n",
       " ('mencurat', 'curat'),\n",
       " ('sangkaanku', 'sangka'),\n",
       " ('mengeringi', 'ering'),\n",
       " ('beranduh', 'anduh'),\n",
       " ('terbaja', 'baja'),\n",
       " ('membegakan', 'bega'),\n",
       " ('terkanjar-kanjar', 'kanjar'),\n",
       " ('pembentrokan', 'bentrok'),\n",
       " ('mengadoi', 'kado'),\n",
       " ('seranggah', 'ranggah'),\n",
       " ('mengilustrasikan', 'ilustrasi'),\n",
       " ('merumrum', 'rumrum'),\n",
       " ('menceritai', 'cita'),\n",
       " ('perembukan', 'rembuk'),\n",
       " ('penjalaan', 'jala'),\n",
       " ('berpetualangan', 'tualang'),\n",
       " ('kandi-kandi', 'kandi'),\n",
       " ('mempersemukakan', 'muka'),\n",
       " ('berjaja', 'jaja'),\n",
       " ('gepengan', 'gepeng'),\n",
       " ('memeres', 'meres'),\n",
       " ('berkial-kial', 'kial'),\n",
       " ('serona', 'rona'),\n",
       " ('mendingini', 'dingin'),\n",
       " ('bulus-bulus', 'bulus'),\n",
       " ('mewaraskan', 'waras'),\n",
       " ('membayang-bayangi', 'bayang'),\n",
       " ('kerevolusioneran', 'revolusioner'),\n",
       " ('mengamen', 'amen'),\n",
       " ('pemantikan', 'manti'),\n",
       " ('pengiaman', 'kiam'),\n",
       " ('pengasung', 'asung'),\n",
       " ('mempertimbangkannya', 'timbang'),\n",
       " ('mendegam-degam', 'degam'),\n",
       " ('sekecil-kecilnya', 'kecil'),\n",
       " ('ketering', 'tering'),\n",
       " ('sejuang', 'juang'),\n",
       " ('diselalukan', 'selalu'),\n",
       " ('berjumbaian', 'jumbai'),\n",
       " ('mengeredep', 'keredep'),\n",
       " ('hina-menghina', 'hina'),\n",
       " ('mengetemukan', 'ketemu'),\n",
       " ('berpendar-pendar', 'pendar'),\n",
       " ('tersenget', 'senget'),\n",
       " ('merajin', 'rajin'),\n",
       " ('berberaso', 'aso'),\n",
       " ('berselumu', 'selumu'),\n",
       " ('pesemaian', 'semai'),\n",
       " ('menunggangkan', 'tunggang'),\n",
       " ('botok-botok', 'botok'),\n",
       " ('mensyarikat', 'syarikat'),\n",
       " ('lapari', 'lapar'),\n",
       " ('belandongan', 'bandongan'),\n",
       " ('berbimbang', 'bimbang'),\n",
       " ('kemburan', 'kembur'),\n",
       " ('menggeleting', 'geleting'),\n",
       " ('tengari', 'tengar'),\n",
       " ('berconteng-conteng', 'conteng'),\n",
       " ('menindikkan', 'tindik'),\n",
       " ('beginian', 'begini'),\n",
       " ('kerampingan', 'ramping'),\n",
       " ('terlengah', 'lengah'),\n",
       " ('mengguntung', 'guntung'),\n",
       " ('berekskursi', 'ekskursi'),\n",
       " ('sedung', 'dung'),\n",
       " ('merebat', 'rebat'),\n",
       " ('sehalus-halusnya', 'halus'),\n",
       " ('berlingkar-lingkar', 'lingkar'),\n",
       " ('bersesah', 'sesah'),\n",
       " ('membindu', 'bindu'),\n",
       " ('terkayuhkan', 'kayuh'),\n",
       " ('kelanting', 'lanting'),\n",
       " ('terbangkang', 'bangkang'),\n",
       " ('perburu', 'buru'),\n",
       " ('persengketakan', 'sengketa'),\n",
       " ('memperusuh', 'rusuh'),\n",
       " ('sebiji-biji', 'biji'),\n",
       " ('memapakan', 'papa'),\n",
       " ('berayunan', 'ayun'),\n",
       " ('terdayuh', 'dayuh'),\n",
       " ('penggaritan', 'garit'),\n",
       " ('berkelun-kelun', 'kelun'),\n",
       " ('mengunyai', 'unyai'),\n",
       " ('bertilamkan', 'tilam'),\n",
       " ('diskors', 'skors'),\n",
       " ('diteguk', 'teguk'),\n",
       " ('menggelitar', 'gelitar'),\n",
       " ('bertakah-takah', 'takah'),\n",
       " ('bergeselan', 'gesel'),\n",
       " ('terdetikkan', 'detik'),\n",
       " ('bertembuk-tembuk', 'tembuk'),\n",
       " ('berbencah-bencah', 'bencah'),\n",
       " ('berlecet', 'lecet'),\n",
       " ('bersaf-saf', 'saf'),\n",
       " ('mempernyatakan', 'nyata'),\n",
       " ('memperlambang', 'perlambang'),\n",
       " ('kependang', 'dang'),\n",
       " ('merimpung', 'rimpung'),\n",
       " ('beragih', 'agih'),\n",
       " ('merepang', 'repang'),\n",
       " ('berketimbungan', 'ketimbung'),\n",
       " ('meraun', 'raun'),\n",
       " ('dirusuhkan', 'rusuh'),\n",
       " ('memagnetkan', 'magnet'),\n",
       " ('saukkanlah', 'sauk'),\n",
       " ('luahkanlah', 'luah'),\n",
       " ('berlangsi', 'langsi'),\n",
       " ('berambur', 'ambur'),\n",
       " ('jemerah', 'jerah'),\n",
       " ('menggeratkan', 'gerat'),\n",
       " ('mencagak', 'cagak'),\n",
       " ('menakari', 'takar'),\n",
       " ('pembantingan', 'banting'),\n",
       " ('bertaraf-taraf', 'taraf'),\n",
       " ('berkumpulan-kumpulan', 'kumpul'),\n",
       " ('mengacapi', 'acap'),\n",
       " ('ketopang', 'topang'),\n",
       " ('penakikan', 'taki'),\n",
       " ('berdeham', 'deham'),\n",
       " ('terkepal', 'kepal'),\n",
       " ('berpenunggang', 'tunggang'),\n",
       " ('mencodak', 'codak'),\n",
       " ('mencuraikan', 'curai'),\n",
       " ('bergelugur', 'gelugur'),\n",
       " ('memperamat-amat', 'amat'),\n",
       " ('berdebat-debatan', 'debat'),\n",
       " ('berembik', 'embik'),\n",
       " ('lekasnya', 'lekas'),\n",
       " ('mempersedap', 'sedap'),\n",
       " ('pembatikan', 'bati'),\n",
       " ('melencit', 'lencit'),\n",
       " ('menenggang', 'tenggang'),\n",
       " ('ingatkannya', 'ingat'),\n",
       " ('keredupan', 'redup'),\n",
       " ('mengobrol', 'obrol'),\n",
       " ('meladamkan', 'ladam'),\n",
       " ('kedampingan', 'damping'),\n",
       " ('pengejatan', 'kejat'),\n",
       " ('mengapokkan', 'apok'),\n",
       " ('bersempuras', 'sempuras'),\n",
       " ('tertebas', 'tebas'),\n",
       " ('bersendar', 'sendar'),\n",
       " ('berbengkok-bengkok', 'bengkok'),\n",
       " ('pemburukan', 'buru'),\n",
       " ('memangkung', 'pangkung'),\n",
       " ('sepedih', 'pedih'),\n",
       " ('sesungguh-sungguhnya', 'sungguh'),\n",
       " ('cementung', 'centung'),\n",
       " ('mengilik-ngilik', 'kilik'),\n",
       " ('meringkai', 'ringkai'),\n",
       " ('menerumkan', 'terum'),\n",
       " ('mencium-cium', 'cium'),\n",
       " ('menyantak', 'santak'),\n",
       " ('mengeruh', 'keruh'),\n",
       " ('berkakak', 'kakak'),\n",
       " ('celebur', 'cebur'),\n",
       " ('seroi', 'sero'),\n",
       " ('pengecap', 'kecap'),\n",
       " ('bertempek-tempek', 'tempek'),\n",
       " ('berkecipak-kecipak', 'kecipak'),\n",
       " ('membisik-bisikkan', 'bisik'),\n",
       " ('memperosokkan', 'perosok'),\n",
       " ('menggarong', 'garong'),\n",
       " ('menginternir', 'internir'),\n",
       " ('menyanyukkan', 'nyanyuk'),\n",
       " ('kehendak-kehendak', 'hendak'),\n",
       " ('bergemeresik', 'gemeresik'),\n",
       " ('menempeli', 'tempel'),\n",
       " ('kehidung-hidungan', 'hidung'),\n",
       " ('penumis', 'tum'),\n",
       " ('memustajabkan', 'mustajab'),\n",
       " ('tersobok', 'sobok'),\n",
       " ('bersambang', 'sambang'),\n",
       " ('bersemburan', 'sembur'),\n",
       " ('melisutkan', 'lisut'),\n",
       " ('bersenggol', 'senggol'),\n",
       " ('keantian', 'anti'),\n",
       " ('menyerantakan', 'seranta'),\n",
       " ('bermukadimah', 'mukadimah'),\n",
       " ('menyandung', 'sandung'),\n",
       " ('menggeritik', 'gitik'),\n",
       " ('serentakan', 'serenta'),\n",
       " ('ternonong-nonong', 'nonong'),\n",
       " ('meretih', 'retih'),\n",
       " ('dahulu-dahulu', 'dahulu'),\n",
       " ('seroma', 'roma'),\n",
       " ('penggabus', 'gabus'),\n",
       " ('mempaung', 'paung'),\n",
       " ('menyeduai', 'duai'),\n",
       " ('kelecat', 'lecat'),\n",
       " ('melangsar', 'langsar'),\n",
       " ('percung', 'cung'),\n",
       " ('sambut-menyambut', 'sambut'),\n",
       " ('berkaum-kaum', 'kaum'),\n",
       " ('menjerahap', 'jerahap'),\n",
       " ('berjuak-juak', 'juak'),\n",
       " ('banting-membanting', 'banting'),\n",
       " ('memperhumai', 'huma'),\n",
       " ('mencahar', 'cahar'),\n",
       " ('erat-erat', 'erat'),\n",
       " ('berbatang-batang', 'batang'),\n",
       " ('menambus', 'tambus'),\n",
       " ('peranggasan', 'ranggas'),\n",
       " ('berselempang', 'selempang'),\n",
       " ('memperedarkan', 'edar'),\n",
       " ('berdompak', 'dompak'),\n",
       " ('melampas', 'lampas'),\n",
       " ('berumun', 'umun'),\n",
       " ('berpancutan', 'pancut'),\n",
       " ('menggeledek', 'geledek'),\n",
       " ('melimaui', 'limau'),\n",
       " ('terbebar', 'bebar'),\n",
       " ('mengunjun', 'unjun'),\n",
       " ('tekeran', 'teker'),\n",
       " ('aling-alingan', 'aling'),\n",
       " ('mereciki', 'recik'),\n",
       " ('mendaifkan', 'daif'),\n",
       " ('tertembus', 'tembus'),\n",
       " ('kefanaan', 'fana'),\n",
       " ('sepemakai', 'maka'),\n",
       " ('menyodokkan', 'sodok'),\n",
       " ('penggorohan', 'goroh'),\n",
       " ('membungkusi', 'bungkus'),\n",
       " ('sentalkan', 'sental'),\n",
       " ('melatahkan', 'latah'),\n",
       " ('buatkannya', 'buat'),\n",
       " ('sekubin', 'kubin'),\n",
       " ('sahutnya', 'sahut'),\n",
       " ('menelungkupkan', 'telungkup'),\n",
       " ('mentamyizkan', 'tamyiz'),\n",
       " ('mencetai-cetai', 'cetai'),\n",
       " ('membedili', 'bedil'),\n",
       " ('memperkarai', 'kara'),\n",
       " ('membengisi', 'bengis'),\n",
       " ('berjalaran', 'jalar'),\n",
       " ('tercogok', 'cogok'),\n",
       " ('bertenang-tenang', 'tenang'),\n",
       " ('kucung', 'cung'),\n",
       " ('memperkononkan', 'konon'),\n",
       " ('setatar', 'tatar'),\n",
       " ('menggerbangkan', 'gerbang'),\n",
       " ('mengupam', 'upam'),\n",
       " ('menyembat', 'sembat'),\n",
       " ('berengap', 'rengap'),\n",
       " ('menanang', 'nanang'),\n",
       " ('penyusur', 'susur'),\n",
       " ('berpancung', 'pancung'),\n",
       " ('mendedak', 'dedak'),\n",
       " ('memperlapang', 'lapang'),\n",
       " ('harga-menghargai', 'harga'),\n",
       " ('menggelingsirkan', 'gelingsir'),\n",
       " ('pengolahraga', 'olahraga'),\n",
       " ('mentarifkan', 'tarif'),\n",
       " ('memperbungakan', 'bunga'),\n",
       " ('bentakan', 'benta'),\n",
       " ('pengharu', 'haru'),\n",
       " ('menggancu', 'gancu'),\n",
       " ('memperkacang', 'kacang'),\n",
       " ('mencongkah', 'mencong'),\n",
       " ('menguparkan', 'upar'),\n",
       " ('pemapaan', 'papa'),\n",
       " ('berjujuh', 'jujuh'),\n",
       " ('pelembungan', 'lembung'),\n",
       " ('berkeredong', 'keredong'),\n",
       " ('secotet', 'cotet'),\n",
       " ('membuaya', 'buaya'),\n",
       " ('keterukkan', 'teruk'),\n",
       " ('memuati', 'muat'),\n",
       " ('pemajangan', 'pajang'),\n",
       " ('menyerendeng', 'serendeng'),\n",
       " ('berpongah-pongah', 'pongah'),\n",
       " ('kipasi', 'kipas'),\n",
       " ('berlanting', 'lanting'),\n",
       " ('memantuni', 'pantun'),\n",
       " ('belit-membelit', 'belit'),\n",
       " ('mencempungi', 'cempung'),\n",
       " ('merunyamkan', 'runyam'),\n",
       " ('pemeroian', 'peroi'),\n",
       " ('membaliki', 'balik'),\n",
       " ('selang-menyelang', 'selang'),\n",
       " ('peninjunya', 'tinju'),\n",
       " ('tersua-sua', 'sua'),\n",
       " ('berbetul', 'betul'),\n",
       " ('memaliskan', 'malis'),\n",
       " ('bergemerencing', 'gemerencing'),\n",
       " ('menggelekek', 'gelekek'),\n",
       " ('memperlabu', 'labu'),\n",
       " ('menyeterika', 'terika'),\n",
       " ('mendiatkan', 'diat'),\n",
       " ('belakai', 'belaka'),\n",
       " ('melusuhkan', 'lusuh'),\n",
       " ('kelentungan', 'kelentung'),\n",
       " ('berlucu', 'lucu'),\n",
       " ('menunggal', 'tunggal'),\n",
       " ('merusuhi', 'rusuh'),\n",
       " ('berkedera', 'kedera'),\n",
       " ('kedeking', 'deking'),\n",
       " ('berlihat-lihatan', 'lihat'),\n",
       " ('menjarum', 'jarum'),\n",
       " ('pesalin', 'salin'),\n",
       " ('berantuk', 'antuk'),\n",
       " ('kemeling', 'ling'),\n",
       " ('tercema', 'cema'),\n",
       " ('membelalak', 'belalak'),\n",
       " ('menggeleser', 'geleser'),\n",
       " ('mengendal', 'endal'),\n",
       " ('mengitalikkan', 'italik'),\n",
       " ('mempersangat', 'sangat'),\n",
       " ('memakati', 'pakat'),\n",
       " ('mengair', 'air'),\n",
       " ('kelemumurnya', 'kelemumur'),\n",
       " ('mengintrospeksi', 'introspeksi'),\n",
       " ('seterikaan', 'terika'),\n",
       " ('membajui', 'baju'),\n",
       " ('membukitkan', 'bukit'),\n",
       " ('mentawan', 'tawan'),\n",
       " ('berpengait', 'kait'),\n",
       " ('menumang', 'tumang'),\n",
       " ('mengembel-embeli', 'embel'),\n",
       " ('terliur', 'liur'),\n",
       " ('menggapai-gapaikan', 'gapai'),\n",
       " ('merakat', 'rakat'),\n",
       " ('menyelendangi', 'selendang'),\n",
       " ('buta-butaan', 'buta'),\n",
       " ('mengalasi', 'alas'),\n",
       " ('mengetengahi', 'tengah'),\n",
       " ('berketel', 'ketel'),\n",
       " ('bergulut-gulut', 'gulut'),\n",
       " ('rakanlah', 'rak'),\n",
       " ('mengarah-arahi', 'arah'),\n",
       " ('memperdayanya', 'daya'),\n",
       " ('menudungi', 'tudung'),\n",
       " ('pekapur', 'kapur'),\n",
       " ('pemarang', 'parang'),\n",
       " ('kasi-kasi', 'kasi'),\n",
       " ('mendungukan', 'dungu'),\n",
       " ('mengembol', 'embol'),\n",
       " ('menjelingkan', 'jeling'),\n",
       " ('menyeluduk', 'seluduk'),\n",
       " ('kapas-kapas', 'kapas'),\n",
       " ('internasionalisme', 'internasional'),\n",
       " ('terhulur-hulur', 'hulur'),\n",
       " ('mereluh', 'ruh'),\n",
       " ('mengogokkan', 'ogok'),\n",
       " ('berkujut', 'kujut'),\n",
       " ('mengegatkan', 'egat'),\n",
       " ('dititikkan', 'titik'),\n",
       " ('terkumpal-kumpal', 'kumpal'),\n",
       " ('terpelancit', 'lancit'),\n",
       " ('beribarat', 'ibarat'),\n",
       " ('berdukung', 'dukung'),\n",
       " ('perlah', 'per'),\n",
       " ('terpercik-percik', 'percik'),\n",
       " ('gagaunya', 'gagau'),\n",
       " ('mengantologikan', 'antologi'),\n",
       " ('sekumin', 'min'),\n",
       " ('menyengkelangkan', 'sengkelang'),\n",
       " ('berdentung-dentung', 'dentung'),\n",
       " ('memanggungkan', 'manggung'),\n",
       " ('berdarmawisata', 'darmawisata'),\n",
       " ('berutas-utas', 'utas'),\n",
       " ('keterakhiran', 'akhir'),\n",
       " ('menyenggak', 'senggak'),\n",
       " ('menyengam', 'sengam'),\n",
       " ('merabik', 'rabik'),\n",
       " ('terdinding', 'dinding'),\n",
       " ('menyumbar', 'sumbar'),\n",
       " ('mengilmukan', 'ilmu'),\n",
       " ('lantung-lantung', 'lantung'),\n",
       " ('merengat-rengat', 'rengat'),\n",
       " ('mengumpak-umpak', 'umpak'),\n",
       " ('melimbur', 'limbur'),\n",
       " ('menyewah', 'sewah'),\n",
       " ('perletupan', 'letup'),\n",
       " ('bertengkar-tengkaran', 'tengkar'),\n",
       " ('tertumbuk-tumbuk', 'tumbuk'),\n",
       " ('kebagusannya', 'bagus'),\n",
       " ('mengantul', 'antul'),\n",
       " ('mengelainkan', 'lain'),\n",
       " ('bertabirkan', 'tabir'),\n",
       " ('peralat', 'alat'),\n",
       " ('mengundai', 'kundai'),\n",
       " ('menguntil', 'until'),\n",
       " ('sayup-menyayup', 'sayup'),\n",
       " ('ringgitnya', 'ringgit'),\n",
       " ('pembusungan', 'busung'),\n",
       " ('menyiahkan', 'siah'),\n",
       " ('pencondongan', 'condong'),\n",
       " ('menceruk', 'ceruk'),\n",
       " ('pekatkan', 'pekat'),\n",
       " ('membelai-belai', 'belai'),\n",
       " ('penginterpretasian', 'interpretasi'),\n",
       " ('kepetengan', 'teng'),\n",
       " ('meranai', 'ranai'),\n",
       " ('kekorok', 'korok'),\n",
       " ('berlanar', 'lanar'),\n",
       " ('penggugatan', 'gugat'),\n",
       " ('mengabjadkan', 'abjad'),\n",
       " ('membulat-bulatkan', 'bulat'),\n",
       " ('mengetakkan', 'ketak'),\n",
       " ('menyira', 'sira'),\n",
       " ('disiahkan', 'siah'),\n",
       " ('berpatik', 'patik'),\n",
       " ('digulung-gulung', 'gulung'),\n",
       " ('berbeting', 'beting'),\n",
       " ('membega-bega', 'bega'),\n",
       " ('pekaca', 'kaca'),\n",
       " ('berteguh-teguhan', 'teguh'),\n",
       " ('mentradisi', 'tradisi'),\n",
       " ('menghirap', 'hirap'),\n",
       " ('memantek', 'pantek'),\n",
       " ('belintan', 'bintan'),\n",
       " ('memuduri', 'pudur'),\n",
       " ('berbebaran', 'bebar'),\n",
       " ('mencopoti', 'copot'),\n",
       " ('bertedengan', 'tedeng'),\n",
       " ('serepuh', 'repuh'),\n",
       " ('mengartukan', 'kartu'),\n",
       " ('terpelekok', 'pelekok'),\n",
       " ('semendal', 'dal'),\n",
       " ('dikasadkan', 'kasad'),\n",
       " ('berungkur-ungkuran', 'ungkur'),\n",
       " ('penjamahan', 'jamah'),\n",
       " ('mempersepsi', 'persepsi'),\n",
       " ('kemaujudan', 'maujud'),\n",
       " ('tertunam', 'tunam'),\n",
       " ('memanahkan', 'manah'),\n",
       " ('penjirus', 'jirus'),\n",
       " ('berkuak-kuak', 'kuak'),\n",
       " ('terpulut', 'pulut'),\n",
       " ('diselakan', 'sela'),\n",
       " ('terlalak-lalak', 'lalak'),\n",
       " ('kemalai', 'kemala'),\n",
       " ('melempit', 'lempit'),\n",
       " ('kekerintingan', 'kerinting'),\n",
       " ('menyudi', 'sudi'),\n",
       " ('kedekak', 'dekak'),\n",
       " ('berbatin', 'batin'),\n",
       " ('menggeleding', 'geleding'),\n",
       " ('telungkupkan', 'telungkup'),\n",
       " ('menimang-nimang', 'timang'),\n",
       " ('menggulaikan', 'gulai'),\n",
       " ('kebungaan', 'bunga'),\n",
       " ('bertunggul', 'tunggul'),\n",
       " ('sekaca', 'kaca'),\n",
       " ('pengemaian', 'mai'),\n",
       " ('mempertenggangkan', 'tenggang'),\n",
       " ('menggidik', 'gidik'),\n",
       " ('menjasai', 'jasa'),\n",
       " ('menjelempah', 'jelempah'),\n",
       " ('berkambi', 'kambi'),\n",
       " ('memperhidup', 'hidup'),\n",
       " ('sekuasanya', 'kuasa'),\n",
       " ('pengembar', 'embar'),\n",
       " ('mempertetapkan', 'tetap'),\n",
       " ('berkoak-koak', 'koak'),\n",
       " ('menobak', 'tobak'),\n",
       " ('garah-garah', 'garah'),\n",
       " ('semerit', 'rit'),\n",
       " ('berkait-kait', 'kait'),\n",
       " ('acak-acakan', 'acak'),\n",
       " ('setumbuk', 'tumbuk'),\n",
       " ('memunggung', 'punggung'),\n",
       " ('berkanjar-kanjar', 'kanjar'),\n",
       " ('membungakan', 'bunga'),\n",
       " ('pengukurnya', 'ukur'),\n",
       " ('mengandak', 'andak'),\n",
       " ('kerutuk', 'rutuk'),\n",
       " ('sepangsa', 'pangsa'),\n",
       " ('terhambur', 'hambur'),\n",
       " ('kegadis-gadisan', 'gadis'),\n",
       " ('mengotek', 'otek'),\n",
       " ('mendayus', 'dayus'),\n",
       " ('mengeritkan', 'kerit'),\n",
       " ('reda-reda', 'reda'),\n",
       " ('pendalih', 'dalih'),\n",
       " ('imbangkan', 'imbang'),\n",
       " ('merenjong', 'renjong'),\n",
       " ('merungkup', 'rungkup'),\n",
       " ('terkampai', 'kampai'),\n",
       " ('kenekadan', 'nekad'),\n",
       " ('cecapan', 'cecap'),\n",
       " ('mengupar', 'upar'),\n",
       " ('berulit', 'ulit'),\n",
       " ('memurus', 'murus'),\n",
       " ('mentaasubi', 'taasub'),\n",
       " ('menelusur', 'telusur'),\n",
       " ('timpa-bertimpa', 'timpa'),\n",
       " ('menggenting', 'genting'),\n",
       " ('pengor', 'kor'),\n",
       " ('bertembak-tembak', 'tembak'),\n",
       " ('menggilapkan', 'gilap'),\n",
       " ('teng-teng', 'teng'),\n",
       " ('mengkolokiumkan', 'kolokium'),\n",
       " ('terjurai-jurai', 'jurai'),\n",
       " ('membulir', 'bulir'),\n",
       " ('mempelepah', 'pelepah'),\n",
       " ('merunjung', 'runjung'),\n",
       " ('aring-aring', 'aring'),\n",
       " ('memial', 'pial'),\n",
       " ('ditebangi', 'tebang'),\n",
       " ('terciar-ciar', 'ciar'),\n",
       " ('kelunturan', 'luntur'),\n",
       " ('mengejai', 'kejai'),\n",
       " ('dismokel', 'smokel'),\n",
       " ('berkian-kian', 'kian'),\n",
       " ('inga-inga', 'inga'),\n",
       " ('mengarau', 'arau'),\n",
       " ('terlongong-longong', 'longong'),\n",
       " ('berus-berus', 'berus'),\n",
       " ('bergamit-gamitan', 'gamit'),\n",
       " ('mengumbah', 'kumbah'),\n",
       " ('dijurus', 'jurus'),\n",
       " ('membanyol', 'banyol'),\n",
       " ('menyerok', 'serok'),\n",
       " ('berpalu-paluan', 'palu'),\n",
       " ('selengkapan', 'lengkap'),\n",
       " ('meleteri', 'leter'),\n",
       " ('terlandai', 'landai'),\n",
       " ('semudik', 'mudik'),\n",
       " ('malaikat-nya', 'malaikat'),\n",
       " ('berpeluh-peluh', 'peluh'),\n",
       " ('melancipkan', 'lancip'),\n",
       " ('mengira-ngirakan', 'ira'),\n",
       " ('dikeritingkan', 'keriting'),\n",
       " ('menimbul-menimbulkan', 'timbul'),\n",
       " ('menohokkan', 'tohok'),\n",
       " ('cinta-mencintai', 'cinta'),\n",
       " ('umpatannya', 'umpat'),\n",
       " ('merenyeh', 'renyeh'),\n",
       " ('mencakarkan', 'cakar'),\n",
       " ('bertempap', 'tempap'),\n",
       " ('berdukun', 'dukun'),\n",
       " ('mendagel', 'dagel'),\n",
       " ('pengantihan', 'antih'),\n",
       " ('memperpotongkan', 'potong'),\n",
       " ('menjenuhkan', 'jenuh'),\n",
       " ('dilupuh', 'lupuh'),\n",
       " ('menjelepak', 'jelepak'),\n",
       " ('mengganjuri', 'ganjur'),\n",
       " ('memberat', 'berat'),\n",
       " ('bertambun-tambun', 'tambun'),\n",
       " ('petandang', 'tandang'),\n",
       " ('memancutkannya', 'pancut'),\n",
       " ('berkerlip-kerlip', 'kerlip'),\n",
       " ('mengkang', 'kang'),\n",
       " ('melondong', 'londong'),\n",
       " ('melenggok-lenggok', 'lenggok'),\n",
       " ('sugi-sugi', 'sugi'),\n",
       " ('kekatung', 'katung'),\n",
       " ('berwaadat', 'waadat'),\n",
       " ('bermurah-murah', 'murah'),\n",
       " ('memikulkan', 'pikul'),\n",
       " ('hisabkan', 'hisab'),\n",
       " ('membisiki', 'bisik'),\n",
       " ('pelopak', 'lopak'),\n",
       " ('berdesau', 'desau'),\n",
       " ('penitipan', 'titip'),\n",
       " ('penyum', 'sum'),\n",
       " ('pekasih', 'kasih'),\n",
       " ('menyobek-nyobekkan', 'sobek'),\n",
       " ('menanjur', 'tanjur'),\n",
       " ('melindap', 'lindap'),\n",
       " ('terlat', 'lat'),\n",
       " ('berpentingan', 'ting'),\n",
       " ('kerekan', 'kere'),\n",
       " ('sekujah', 'jah'),\n",
       " ('mempercerataikan', 'ceratai'),\n",
       " ('terkulai-kulai', 'kulai'),\n",
       " ('mendingan', 'mending'),\n",
       " ('rambaian', 'rambai'),\n",
       " ('terperanyak', 'anyak'),\n",
       " ('kelemut', 'lut'),\n",
       " ('bergandung', 'gandung'),\n",
       " ('sebabat', 'babat'),\n",
       " ('melontai', 'lontai'),\n",
       " ('tergocoh-gocoh', 'gocoh'),\n",
       " ('bermemek-memek', 'memek'),\n",
       " ('keserabutan', 'serabut'),\n",
       " ('risauan', 'risau'),\n",
       " ('menjenazahkan', 'jenazah'),\n",
       " ('memberak', 'berak'),\n",
       " ('layeran', 'layer'),\n",
       " ('mengesatkan', 'kesat'),\n",
       " ('terbalak-balak', 'balak'),\n",
       " ('pasukan-pasukannya', 'pasu'),\n",
       " ('dirimbunkan', 'rimbun'),\n",
       " ('sepelan', 'sepel'),\n",
       " ('niatlah', 'niat'),\n",
       " ('menonjok', 'tonjok'),\n",
       " ('menggemburi', 'gembur'),\n",
       " ('telepa', 'tepa'),\n",
       " ('mendobelkan', 'dobel'),\n",
       " ('terbidik-bidik', 'bidik'),\n",
       " ('penumbukan', 'tumbu'),\n",
       " ('kehakikian', 'hakiki'),\n",
       " ('berpenggalan', 'galan'),\n",
       " ('merompongkan', 'rompong'),\n",
       " ('kekeng', 'keng'),\n",
       " ('menikis', 'nik'),\n",
       " ('melenjakan', 'lenja'),\n",
       " ('terliput', 'liput'),\n",
       " ('menjejalkan', 'jejal'),\n",
       " ('memperselangi', 'langi'),\n",
       " ('mengumaikan', 'kumai'),\n",
       " ('memelonco', 'pelonco'),\n",
       " ('bersilir-silir', 'silir'),\n",
       " ('menyetat', 'setat'),\n",
       " ('diperdomi', 'dom'),\n",
       " ('bercemabai', 'caba'),\n",
       " ('tersingahak', 'singahak'),\n",
       " ('memperkonon', 'konon'),\n",
       " ('deram-deram', 'deram'),\n",
       " ('melejang-lejangkan', 'lejang'),\n",
       " ('menjujuri', 'jujur'),\n",
       " ('menggerinjam', 'gerinjam'),\n",
       " ('berantaraan', 'antara'),\n",
       " ('merapai', 'rapai'),\n",
       " ('menguber-uber', 'uber'),\n",
       " ('teraduk-aduk', 'aduk'),\n",
       " ('kesiuman', 'siuman'),\n",
       " ('bertambul', 'tambul'),\n",
       " ('berconggok', 'conggok'),\n",
       " ('bersenggat-senggat', 'senggat'),\n",
       " ('pecomberan', 'comberan'),\n",
       " ('ulak-ulak', 'ulak'),\n",
       " ('menyidaikan', 'sidai'),\n",
       " ('pedarat', 'darat'),\n",
       " ('beruntai', 'unta'),\n",
       " ('runcing-runcing', 'runcing'),\n",
       " ('memenyapkan', 'penyap'),\n",
       " ('kemurahan hati', 'murah hati'),\n",
       " ('berserama', 'serama'),\n",
       " ('menyimpangi', 'simpang'),\n",
       " ('berpakansi', 'pakansi'),\n",
       " ('mempermadukan', 'madu'),\n",
       " ('melantakkan', 'lantak'),\n",
       " ('kuisan', 'kuis'),\n",
       " ('berladam', 'ladam'),\n",
       " ('binti-binti', 'binti'),\n",
       " ('berlipat-lipatan', 'lipat'),\n",
       " ('tergelak-gelak', 'gelak'),\n",
       " ('terdemam-demam', 'demam'),\n",
       " ('saukkan', 'sauk'),\n",
       " ('langganannya', 'langgan'),\n",
       " ('serenteng', 'renteng'),\n",
       " ('mengagut-agut', 'agut'),\n",
       " ('pengerawan', 'rawan'),\n",
       " ('memanggangkan', 'panggang'),\n",
       " ('menokohi', 'tokoh'),\n",
       " ('menugalkan', 'tugal'),\n",
       " ('dialang-alang', 'alang'),\n",
       " ('berkurik', 'kurik'),\n",
       " ('menggarut', 'garut'),\n",
       " ('menghijrahkan', 'hijrah'),\n",
       " ('selebih-lebihnya', 'lebih'),\n",
       " ('mengantuknya', 'antuk'),\n",
       " ('menggurit', 'gurit'),\n",
       " ('menyendukkan', 'senduk'),\n",
       " ('penyangka', 'sangka'),\n",
       " ('dipermainnya', 'main'),\n",
       " ('terangin', 'angin'),\n",
       " ('memperadapkan', 'adap'),\n",
       " ('bungkalan', 'bungkal'),\n",
       " ('pemompaan', 'pompa'),\n",
       " ('merelang', 'relang'),\n",
       " ('menyebaikan', 'sebai'),\n",
       " ('mengantungi', 'kantung'),\n",
       " ('dicucuk-cucuk', 'cucuk'),\n",
       " ('berdenai', 'dena'),\n",
       " ('menyenggayuti', 'senggayut'),\n",
       " ('bertokohkan', 'tokoh'),\n",
       " ('mengalati', 'alat'),\n",
       " ('terigau-igau', 'igau'),\n",
       " ('kelengangan', 'lengang'),\n",
       " ('bergolok', 'golok'),\n",
       " ('menjamakkan', 'jamak'),\n",
       " ('meneluskan', 'telus'),\n",
       " ('mempreskripsikan', 'preskripsi'),\n",
       " ('bertelimpuh', 'telimpuh'),\n",
       " ('mengernyutkan', 'kernyut'),\n",
       " ('terserai', 'serai'),\n",
       " ('dibabarkan', 'babar'),\n",
       " ('mengesup', 'kesup'),\n",
       " ('dimanja-manjakan', 'manja'),\n",
       " ('kesatkan', 'kesat'),\n",
       " ('meronyeh-ronyeh', 'ronyeh'),\n",
       " ('berluruhan', 'luruh'),\n",
       " ('memperdekap', 'dekap'),\n",
       " ('embel-embelan', 'embel'),\n",
       " ('menginjeksi', 'injeksi'),\n",
       " ('kementerengan', 'mentereng'),\n",
       " ('lelembu', 'lembu'),\n",
       " ('terjelapak', 'jelapak'),\n",
       " ('bergembut-gembut', 'gembut'),\n",
       " ('sehilir', 'hilir'),\n",
       " ('menjarang', 'jarang'),\n",
       " ('merawankan', 'merawan'),\n",
       " ('corek-corek', 'corek'),\n",
       " ('melipit', 'lipit'),\n",
       " ('pendirusan', 'dirus'),\n",
       " ('sejulung', 'julung'),\n",
       " ('menggelugut', 'gelugut'),\n",
       " ('pemunggai', 'punggai'),\n",
       " ('bersuten', 'suten'),\n",
       " ('bergamat', 'gamat'),\n",
       " ('menumbuhi', 'tumbuh'),\n",
       " ('ketabiban', 'tabib'),\n",
       " ('memasalahkan', 'masalah'),\n",
       " ('menghadi', 'had'),\n",
       " ('keujian', 'uji'),\n",
       " ('menukar-nukar', 'tukar'),\n",
       " ('penoda', 'noda'),\n",
       " ('bertiras-tiras', 'tiras'),\n",
       " ('persuaan', 'sua'),\n",
       " ('kekantung', 'kantung'),\n",
       " ('tersorong-sorong', 'sorong'),\n",
       " ('membedak', 'bedak'),\n",
       " ('kirim-mengirim', 'kirim'),\n",
       " ('mencubit-cubit', 'cubit'),\n",
       " ('telengkoh', 'tengkoh'),\n",
       " ('perongos', 'rongos'),\n",
       " ('pendukuhan', 'dukuh'),\n",
       " ('menekapkan', 'tekap'),\n",
       " ('tekenan', 'teken'),\n",
       " ('seyojana', 'yojana'),\n",
       " ('kupaskan', 'kupas'),\n",
       " ('tekapkan', 'tekap'),\n",
       " ('ditunjuk-tunjuk', 'tunjuk'),\n",
       " ('selepoh', 'lepoh'),\n",
       " ('meroyalkan', 'royal'),\n",
       " ('penjentik', 'jentik'),\n",
       " ('sentaknya', 'sentak'),\n",
       " ('seingat-ingat', 'ingat'),\n",
       " ('serenyuk', 'renyuk'),\n",
       " ('gelentar', 'gentar'),\n",
       " ('menggeletuk', 'geletuk'),\n",
       " ('bersenggayut', 'senggayut'),\n",
       " ('memarit', 'parit'),\n",
       " ('pelindis', 'lindis'),\n",
       " ('tergegar', 'gegar'),\n",
       " ('ambil-mengambil', 'ambil'),\n",
       " ('menisi', 'tis'),\n",
       " ('meranyah', 'ranyah'),\n",
       " ('berkedal', 'kedal'),\n",
       " ('pengorganisasian', 'organisasi'),\n",
       " ('pandang-memandang', 'pandang'),\n",
       " ('ditisik', 'tisik'),\n",
       " ('teruntil-until', 'until'),\n",
       " ('optimumkan', 'optimum'),\n",
       " ('menitiki', 'titik'),\n",
       " ('menggelepai', 'gelepai'),\n",
       " ('diambin', 'ambin'),\n",
       " ('kepilkan', 'kepil'),\n",
       " ('menggusari', 'gusar'),\n",
       " ('kebalkan', 'kebal'),\n",
       " ('dicelakakan', 'celaka'),\n",
       " ('merahapkan', 'rahap'),\n",
       " ('memperligat', 'ligat'),\n",
       " ('penjabatan', 'jabat'),\n",
       " ('sulapan', 'sulap'),\n",
       " ('igaukan', 'igau'),\n",
       " ('tanzilan', 'tanzil'),\n",
       " ('terkitai-kitai', 'kita'),\n",
       " ('bercatur', 'catur'),\n",
       " ('berkasur', 'kasur'),\n",
       " ('sejahat-jahat', 'jahat'),\n",
       " ('semulut', 'mulut'),\n",
       " ('menggencatkan', 'gencat'),\n",
       " ('melokalisasikan', 'lokalisasi'),\n",
       " ('lihatan', 'lihat'),\n",
       " ('merelevankan', 'relevan'),\n",
       " ('perajut', 'rajut'),\n",
       " ('menyambitkan', 'sambit'),\n",
       " ('memincuk', 'pincuk'),\n",
       " ('bergelingsir', 'gelingsir'),\n",
       " ('bertentu-tentu', 'tentu'),\n",
       " ('keloloh', 'loloh'),\n",
       " ('terayak-ayak', 'ayak'),\n",
       " ('kekalapan', 'kalap'),\n",
       " ('kemesuman', 'mesum'),\n",
       " ('hirupan', 'hirup'),\n",
       " ('pembirokrasian', 'birokrasi'),\n",
       " ('pemahal', 'mahal'),\n",
       " ('bercoreng-coreng', 'coreng'),\n",
       " ('mematah-matah', 'patah'),\n",
       " ('perat-perat', 'perat'),\n",
       " ('menjelau', 'jelau'),\n",
       " ('berbawaan', 'bawa'),\n",
       " ('renyuk-renyuk', 'renyuk'),\n",
       " ('melelaikan', 'lelai'),\n",
       " ('pengamat-amatan', 'amat'),\n",
       " ('menyapu-nyapu', 'sapu'),\n",
       " ('dikunyah-kunyah', 'kunyah'),\n",
       " ('teragung-agung', 'agung')]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_same[-1000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 (True, 'mmmnggenjak-enjak', 'enjak')\n",
      "1 (True, 'mmmeengenjak-enjak', 'enjak')\n",
      "2 (True, 'mennngenjak-enjak', 'enjak')\n",
      "3 (True, 'mngenjak-enjak', 'enjak')\n",
      "4 (True, 'mmmngenjak-enjak', 'enjak')\n",
      "5 (True, 'mnngenjak-enjak', 'enjak')\n",
      "6 (True, 'menngenjak-enjak', 'enjak')\n",
      "7 (True, 'mngenjak-enjak', 'enjak')\n",
      "8 (False, 'mengenjak-enjak', 'enjak')\n",
      "9 (True, 'menggenjak-enjak', 'enjak')\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(i, augment_left_right('mengenjak-enjak', 'enjak'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mmmeeengenjak-enjak<<>>enjak'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'<<>>'.join(augment_left_right('mengenjak-enjak', 'enjak')[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 membidai<<>>bidai\n",
      "1 membidai<<>>bidai\n",
      "2 membidai<<>>bidai\n",
      "3 membdai<<>>bdai\n",
      "4 membidai<<>>bidai\n",
      "5 membidai<<>>bidai\n",
      "6 membidai<<>>bidai\n",
      "7 membidai<<>>bidai\n",
      "8 membidai<<>>bidai\n",
      "9 membidai<<>>bidai\n",
      "10 membidai<<>>bidai\n",
      "11 membidai<<>>bidai\n",
      "12 membidai<<>>bidai\n",
      "13 membidai<<>>bidai\n",
      "14 membidai<<>>bidai\n",
      "15 membidai<<>>bidai\n",
      "16 membidai<<>>bidai\n",
      "17 membidai<<>>bidai\n",
      "18 membdai<<>>bdai\n",
      "19 membidai<<>>bidai\n",
      "20 membidai<<>>bidai\n",
      "21 membidai<<>>bidai\n",
      "22 membidai<<>>bidai\n",
      "23 membidai<<>>bidai\n",
      "24 membidai<<>>bidai\n",
      "25 membidai<<>>bidai\n",
      "26 membidai<<>>bidai\n",
      "27 membidai<<>>bidai\n",
      "28 membidai<<>>bidai\n",
      "29 membidai<<>>bidai\n",
      "30 membidai<<>>bidai\n",
      "31 membidai<<>>bidai\n",
      "32 membidai<<>>bidai\n",
      "33 membidai<<>>bidai\n",
      "34 membidai<<>>bidai\n",
      "35 membidai<<>>bidai\n",
      "36 membidai<<>>bidai\n",
      "37 membidai<<>>bidai\n",
      "38 membidai<<>>bidai\n",
      "39 membidai<<>>bidai\n",
      "40 membidai<<>>bidai\n",
      "41 membdai<<>>bdai\n",
      "42 membidai<<>>bidai\n",
      "43 membidai<<>>bidai\n",
      "44 membidai<<>>bidai\n",
      "45 membidai<<>>bidai\n",
      "46 membidai<<>>bidai\n",
      "47 membidai<<>>bidai\n",
      "48 membidai<<>>bidai\n",
      "49 membidai<<>>bidai\n"
     ]
    }
   ],
   "source": [
    "for i in range(50):\n",
    "    print(i, '<<>>'.join(augment_left_right('membidai', 'bidai', augment_hubungan = False, \n",
    "                                           p_augment_word = 0.85)[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 198320/198320 [00:35<00:00, 5526.65it/s]\n"
     ]
    }
   ],
   "source": [
    "for n in tqdm(not_same):\n",
    "    for _ in range(3):\n",
    "        try:\n",
    "            \n",
    "            l = augment_left_right(n[0], n[1], augment_hubungan = False, p_augment_word = 0.85)\n",
    "            if l[0]:\n",
    "                combined.append('<<>>'.join(l[1:]))\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    for _ in range(7):\n",
    "        try:\n",
    "            l = augment_left_right(*n)\n",
    "            if l[0]:\n",
    "                combined.append('<<>>'.join(l[1:]))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1228060"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(False, 'agung', 'agung')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augment_hujung('agung')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 63838/63838 [00:05<00:00, 11356.69it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['jauhar',\n",
       " 'lecut',\n",
       " 'perkhatanan',\n",
       " 'pegas',\n",
       " 'seranah',\n",
       " 'terka',\n",
       " 'maki',\n",
       " 'bersempudal',\n",
       " 'cacat',\n",
       " 'gadai']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled = []\n",
    "for w in tqdm(malaya.dictionary.bahasa.dbp.words | malaya.dictionary.bahasa.cambridge_words.words | malaya.dictionary.bahasa.kamus_dewan.words | malaya.dictionary.bahasa.words.words):\n",
    "    s = sastrawi.stem(w)\n",
    "    sampled.append(s)\n",
    "    \n",
    "sampled[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41817"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_sampled = set(sampled + [i[1] for i in not_same] + random.sample())\n",
    "len(new_sampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 41817/41817 [01:24<00:00, 494.48it/s]\n"
     ]
    }
   ],
   "source": [
    "for r in tqdm(new_sampled):\n",
    "    a = []\n",
    "    try:\n",
    "        for _ in range(20):\n",
    "            l = augment_hujung(r)\n",
    "            if l[0]:\n",
    "                a.append('<<>>'.join(l[1:]))\n",
    "    except:\n",
    "        pass\n",
    "    a = list(set(a))\n",
    "    combined.extend(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2773954"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = set(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2035197"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(combined))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "for c in combined:\n",
    "    if '<<>>' in c:\n",
    "        l, r = c.split('<<>>')\n",
    "        filtered_left.append(l)\n",
    "        filtered_right.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(filtered_left, filtered_right, test_size = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['pstaann',\n",
       "  'bradey',\n",
       "  'yakkun',\n",
       "  'Tsukiatteiru',\n",
       "  'botor',\n",
       "  'diamarkaannn',\n",
       "  'Brsepada',\n",
       "  'Changanassery',\n",
       "  'rightly',\n",
       "  'dabudabu'],\n",
       " ['psta',\n",
       "  'bradey',\n",
       "  'yakkun',\n",
       "  'tsukiatteiru',\n",
       "  'botor',\n",
       "  'amar',\n",
       "  'sepada',\n",
       "  'changanassery',\n",
       "  'rightly',\n",
       "  'dabudabu'])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X[:10], train_Y[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train_noisy_stem.json', 'w') as fopen:\n",
    "    json.dump({'X': train_X, 'Y': train_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test_noisy_stem.json', 'w') as fopen:\n",
    "    json.dump({'X': test_X, 'Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1831677, 203520)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_X), len(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(filtered_left + filtered_right))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import youtokentome as yttm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "stemmer_bpe = yttm.BPE.train(data='text.txt', vocab_size=1000, model='stemmer-v2.model',\n",
    "              pad_id=0, unk_id=2, bos_id=3, eos_id=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
